{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "%run -n main.py"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# path = join_path(DATA_DIR, DATASET)\n",
    "# !mkdir -p {path}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for name in DATASETS:\n",
    "#     file = CORUS_FILES[name]\n",
    "#     if name == AE:\n",
    "#         path = [join_path(CORUS_DATA_DIR, _) for _ in file]\n",
    "#     else:\n",
    "#         path = join_path(CORUS_DATA_DIR, file)\n",
    "    \n",
    "#     records = DATASETS[name](path)\n",
    "#     records = log_progress(records, desc=name)\n",
    "#     items = as_jsons(records)\n",
    "#     lines = format_jl(items)\n",
    "#     path = join_path(DATA_DIR, DATASET, name + JL + GZ)\n",
    "#     dump_gz_lines(lines, path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "datasets = {}\n",
    "for name in log_progress(DATASETS):\n",
    "    path = join_path(DATA_DIR, DATASET, name + JL + GZ)\n",
    "    lines = load_gz_lines(path)\n",
    "    items = parse_jl(lines)\n",
    "    datasets[name] = list(from_jsons(items, Sim))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for scheme in SCHEMES:\n",
    "#     path = join_path(DATA_DIR, scheme.name)\n",
    "#     !mkdir -p {path}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dir = RUSVEC_DATA_DIR\n",
    "# # !wget http://vectors.nlpl.eu/repository/11/180.zip -O {dir}/ruscorpora_upos_cbow_300_20_2019.zip\n",
    "# # !unzip {dir}/ruscorpora_upos_cbow_300_20_2019.zip -d {dir}/ruscorpora_upos_cbow_300_20_2019\n",
    "# # !rm {dir}/ruscorpora_upos_cbow_300_20_2019.zip {dir}/ruscorpora_upos_cbow_300_20_2019/model.txt\n",
    "\n",
    "# # !wget http://vectors.nlpl.eu/repository/11/182.zip -O {dir}/ruwikiruscorpora_upos_skipgram_300_2_2019.zip\n",
    "# # !unzip {dir}/ruwikiruscorpora_upos_skipgram_300_2_2019.zip -d {dir}/ruwikiruscorpora_upos_skipgram_300_2_2019\n",
    "# # !rm {dir}/ruwikiruscorpora_upos_skipgram_300_2_2019.zip {dir}/ruwikiruscorpora_upos_skipgram_300_2_2019/model.txt\n",
    "\n",
    "# # !wget http://vectors.nlpl.eu/repository/11/185.zip -O {dir}/tayga_upos_skipgram_300_2_2019.zip\n",
    "# # !unzip {dir}/tayga_upos_skipgram_300_2_2019.zip -d {dir}/tayga_upos_skipgram_300_2_2019\n",
    "# # !rm {dir}/tayga_upos_skipgram_300_2_2019.zip {dir}/tayga_upos_skipgram_300_2_2019/model.txt\n",
    "\n",
    "# # !wget http://vectors.nlpl.eu/repository/11/187.zip -O {dir}/tayga_none_fasttextcbow_300_10_2019.zip\n",
    "# # !unzip {dir}/tayga_none_fasttextcbow_300_10_2019.zip -d {dir}/tayga_none_fasttextcbow_300_10_2019\n",
    "# # !rm {dir}/tayga_none_fasttextcbow_300_10_2019.zip\n",
    "\n",
    "# # !wget https://rusvectores.org/static/models/rusvectores4/fasttext/araneum_none_fasttextcbow_300_5_2018.tgz -O {dir}/araneum_none_fasttextcbow_300_5_2018.tgz\n",
    "# # !mkdir {dir}/araneum_none_fasttextcbow_300_5_2018\n",
    "# # !tar xzvf {dir}/araneum_none_fasttextcbow_300_5_2018.tgz -C {dir}/araneum_none_fasttextcbow_300_5_2018\n",
    "# # !rm {dir}/araneum_none_fasttextcbow_300_5_2018.tgz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# for scheme in SCHEMES:\n",
    "#     log(scheme.name)\n",
    "#     model = scheme.load()\n",
    "#     for dataset_name in datasets:\n",
    "#         records = datasets[dataset_name]\n",
    "#         records = log_progress(records, desc=dataset_name)\n",
    "#         records = map_model(model, records, scheme.tagged)\n",
    "#         items = as_jsons(records)\n",
    "#         lines = format_jl(items)\n",
    "#         path = join_path(DATA_DIR, scheme.name, dataset_name + JL + GZ)\n",
    "#         dump_gz_lines(lines, path)\n",
    "        \n",
    "#         records = datasets[dataset_name]\n",
    "#         for record in records[:1000]:  # measure get perf\n",
    "#             model.get(record.a)\n",
    "            \n",
    "#         path = join_path(DATA_DIR, scheme.name, STATS + JSON)\n",
    "#         text = format_json(scheme.stats.as_json)\n",
    "#         dump_text(text, path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "stats = {}\n",
    "for scheme in SCHEMES:\n",
    "    path = join_path(DATA_DIR, scheme.name, STATS + JSON)\n",
    "    text = load_text(path)\n",
    "    data = parse_json(text)\n",
    "    stats[scheme.name] = Stats.from_json(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "scores = {}\n",
    "keys = [\n",
    "    (scheme.name, dataset)\n",
    "    for scheme in SCHEMES\n",
    "    for dataset in DATASETS\n",
    "]\n",
    "for model, dataset in log_progress(keys):\n",
    "    targets = datasets[dataset]\n",
    "\n",
    "    path = join_path(DATA_DIR, model, dataset + JL + GZ)\n",
    "    lines = load_gz_lines(path)\n",
    "    items = parse_jl(lines)\n",
    "    preds = list(from_jsons(items, Sim))\n",
    "\n",
    "    score = SCORES[dataset](preds, targets)\n",
    "    scores[model, dataset] = score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "table = report_table(scores, stats, DATASETS, SCHEMES)\n",
    "html = format_report(table, DATASETS)\n",
    "HTML(html)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "html = format_github_report1(table)\n",
    "patch_readme(EMB1, html, README)\n",
    "patch_readme(EMB1, html, NAVEC_README)\n",
    "HTML(html)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "html = format_github_report2(table, DATASETS)\n",
    "patch_readme(EMB2, html, README)\n",
    "patch_readme(EMB2, html, NAVEC_README)\n",
    "HTML(html)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
